import requests
import re
import pymysql
import hashlib
import datetime


class Demo(object):
    def __init__(self):
        self.host = '127.0.0.1'
        self.db = 'app_mark'
        self.user = 'root'
        self.passwd = '123456'
        self.charset = 'utf8mb4'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36',
        }
        self.url = 'http://www.lvmama.com/'
        self.channel_link = [
            'http://s.lvmama.com/group/H13K110000?keyword=%E6%99%AE%E5%90%89%E5%B2%9B&k=0#list',  # 海岛
            'http://s.lvmama.com/route/H13K310000?keyword=%E6%96%B0%E5%8A%A0%E5%9D%A1&k=0#list',  # 东南亚
            'http://s.lvmama.com/route/H13K310000?keyword=%E9%A6%99%E6%B8%AF&k=0#list',  # 中国港澳台
            'http://s.lvmama.com/group/H13K110000?keyword=%E8%BF%AA%E6%8B%9C&k=0#list',  # 迪拜
            'http://s.lvmama.com/group/C262H13K110000?keyword=%E4%BF%84%E7%BD%97%E6%96%AF&tabType=group#list',  # 俄罗斯
            'http://s.lvmama.com/group/H13K110000Y4?keyword=%E8%B6%8A%E5%8D%97#list#list',  # 越南
            'http://s.lvmama.com/group/C265H13K110000?keyword=%E6%B3%95%E5%9B%BD&tabType=group#list%22',  # 法瑞意德
            'http://s.lvmama.com/group/H13K110000?keyword=%E5%B7%B4%E5%8E%98%E5%B2%9B&k=0#list',  # 巴厘岛
            'http://s.lvmama.com/route/H13K310000?keyword=%E6%97%A5%E6%9C%AC&k=0#list',  # 日本
            'http://s.lvmama.com/route/H13K310000?keyword=%E6%AC%A7%E6%B4%B2&k=0#list',  # 欧美
            'http://s.lvmama.com/route/H13K440100?keyword=%E6%96%B0%E5%8A%A0%E5%9D%A1&k=0#list',  # 新加坡
            'http://s.lvmama.com/route/H13K310000?keyword=%E9%A6%99%E6%B8%AF&k=0#list',  # 香港
            'http://s.lvmama.com/route/H13K310000?keyword=%E6%BE%B3%E6%B4%B2&k=0#list',  # 澳洲
            'http://s.lvmama.com/route/H13K310000?keyword=%E6%B3%B0%E5%9B%BD&k=0#list',  # 泰国
            'http://s.lvmama.com/route/H13K440300?keyword=%E4%B8%89%E4%BA%9A&k=0#list',  # 三亚
            'http://s.lvmama.com/route/H13K440300P2?keyword=%E4%B8%89%E4%BA%9A&tabType=route350',  # 三亚p2
            'http://s.lvmama.com/route/H13K440300P3?keyword=%E4%B8%89%E4%BA%9A&tabType=route350',  # 三亚p3
            'http://s.lvmama.com/route/H13K440300P4?keyword=%E4%B8%89%E4%BA%9A&tabType=route350',  # 三亚p4
            'http://s.lvmama.com/route/H13K440300?keyword=%E5%8E%A6%E9%97%A8&k=0#list',  # 厦门
            'http://s.lvmama.com/route/H13K440300?keyword=%E5%B9%BF%E4%B8%9C&k=0#list',  # 广东
            'http://s.lvmama.com/route/H13K440300?keyword=%E4%BA%91%E5%8D%97&k=0#list',  # 云南
            'http://s.lvmama.com/route/H13K440300?keyword=%E4%B8%8A%E6%B5%B7&k=0#list',  # 上海
            'http://s.lvmama.com/route/H13K440300?keyword=%E8%A5%BF%E5%AE%89&k=0#list',  # 西安
            'http://s.lvmama.com/route/H13K440300?keyword=%E6%88%90%E9%83%BD&k=0#list',  # 成都
            'http://s.lvmama.com/route/H13K440300?keyword=%E5%90%89%E6%9E%97&k=0#list',  # 吉林
            'http://s.lvmama.com/route/H13K440300?keyword=%E8%A5%BF%E5%8C%97&k=0#list',  # 西北
            'http://s.lvmama.com/scenictour/K110000?keyword=%E5%8C%97%E4%BA%AC&k=0#list',  # 北京
            'http://s.lvmama.com/scenictour/K110000?keyword=%E5%B1%B1%E4%B8%9C&k=0#list',  # 山东
            'http://s.lvmama.com/scenictour/K110000?keyword=%E5%B1%B1%E8%A5%BF&k=0#list',  # 山西
            'http://s.lvmama.com/scenictour/K110000?keyword=%E6%B2%B3%E5%8C%97&k=0#list',  # 河北
            'http://s.lvmama.com/scenictour/K110000?keyword=%E8%BE%BD%E5%AE%81&k=0#list',  # 辽宁
            ]
        self.channel_name = [
            '海岛',
            '东南亚',
            '中国港澳台',
            '迪拜',
            '俄罗斯',
            '越南',
            '法瑞意德',
            '巴厘岛',
            '日本',
            '欧洲',
            '新加坡',
            '香港',
            '澳洲',
            '泰国',
            '三亚',
            '三亚p2',
            '三亚p3',
            '三亚p4',
            '厦门',
            '广东',
            '云南',
            '上海',
            '西安',
            '成都',
            '吉林',
            '西北',
            '北京',
            '山东',
            '山西',
            '河北',
            '辽宁',
        ]

    def get_html(self, url):
        response = requests.get(url, headers=self.headers)
        response.encoding = response.apparent_encoding
        html = response.text
        return html

    def get_data(self):
        # 首页抓取
        # html = self.get_html(self.url)
        # datas = re.findall('<li data-mmurl=.*?<div class="footLink">', html, re.S)[0]
        # lis = re.findall('(<li data-mmurl=.*?</li>)', datas, re.S)
        # for li in lis:
        #     # detail_url = re.findall('<li data-mmurl="(.*?)"', li, re.S)  # 详情页app链接
        #     detail_url = re.findall('href="(.*?)"', li, re.S)[0]  # 详情页网页链接
        #     self.save_data(detail_url)
        # print(datas)

        # 频道抓取
        urls = []
        # 正则匹配链接
        for index, channel in enumerate(self.channel_link):
            html = self.get_html(channel)
            divs = re.findall('<div class="product-left".*<div class="paging orangestyle"', html, re.S)[0]
            divs = re.findall('<div class="product-section">.*?</div>', divs, re.S)
            for div in divs:
                print(self.channel_name[index])
                url = re.findall('<a href="(.*?)"', div, re.S)[0]
                self.save_data(url)

    def save_data(self, url):
        print(url)
        hkey = hashlib.md5(url.encode(encoding='utf-8')).hexdigest()
        sitename = '驴妈妈旅游'
        lasttime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        tag = '0'
        list_sql = [url, hkey, tag, sitename, lasttime]
        con = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset=self.charset)
        cur = con.cursor()
        sql = 'insert into gly(link, hkey, tag, sitename, lasttime) values (%s, %s, %s, %s, %s)'
        try:
            cur.execute(sql, list_sql)
            print('insert success')
        except Exception as e:
            con.rollback()
            print('error~', e)
        else:
            con.commit()
        cur.close()
        con.close()


if __name__ == '__main__':
    demo = Demo()
    demo.get_data()
